1 Data preparation

1.1 Data wrangling

1.1.1 Load data

### Load data
df_dtw_zoom = read_csv("data/dtw_distance_zoom.csv") %>% 
  rename(target = target_2) %>%
  select(comparison_id, pair, target, average_distance, average_distance_xyz, hands_dtw) %>%
  mutate(dataset = "zoom",
         pair = as.factor(pair),
         target = as.factor(target))

df_sim_coding_zoom = read_csv("data/similarity_coding_zoom.csv") %>% 
  rename(position = location) %>% 
  mutate(total_similarity = handshape + movement + orientation + position,
         ### binary coding
         handshape_bin = ifelse(handshape >= 4, 1, 0),
         movement_bin = ifelse(movement >= 4, 1, 0),
         orientation_bin = ifelse(orientation >= 4, 1, 0),
         position_bin = ifelse(position >= 4, 1, 0),
         n_features = handshape_bin + movement_bin + orientation_bin + position_bin,
         handshape_bin = factor(handshape_bin, 
                                levels = c(0, 1), 
                                labels = c("not similar", "similar")),
         movement_bin = factor(movement_bin,
                                levels = c(0, 1), 
                                labels = c("not similar", "similar")),
         orientation_bin = factor(orientation_bin,
                                   levels = c(0, 1), 
                                   labels = c("not similar", "similar")),
         position_bin = factor(position_bin,
                                levels = c(0, 1), 
                                labels = c("not similar", "similar")),
         ### z-score
         handshape_z = scale(handshape)[,1],
         movement_z = scale(movement)[,1],
         orientation_z = scale(orientation)[,1],
         position_z = scale(position)[,1],
         total_similarity_z = scale(total_similarity)[,1]) %>% 
  select(-notes)

### combine dtw and coding data for zoom dataset
df_dtw_zoom = right_join(df_dtw_zoom, df_sim_coding_zoom)

### convert to long format for data visualization
df_dtw_zoom_long = df_dtw_zoom %>%
  pivot_longer(cols = c("handshape", "movement", "orientation", "position"), 
               names_to = "feature", 
               values_to = "similarity") %>% 
  select(-ends_with("_z"), -ends_with("_bin"))

df_dtw_zoom_long_bin = df_dtw_zoom %>%
  pivot_longer(cols = c("handshape_bin", "movement_bin", "orientation_bin", "position_bin"), 
               names_to = "feature", 
               values_to = "similarity_bin") %>% 
  mutate(feature = factor(sub("_bin", "", feature),
                          levels = c("handshape", "movement", "orientation", "position"))) %>% 
  select(-handshape, -movement, -orientation, -position, -ends_with("_z"))

df_dtw_zoom_long_z = df_dtw_zoom %>%
  pivot_longer(cols = c("handshape_z", "movement_z", "orientation_z", "position_z"), 
               names_to = "feature", 
               values_to = "similarity_z") %>% 
  mutate(feature = factor(sub("_z", "", feature),
                          levels = c("handshape", "movement", "orientation", "position"))) %>% 
  select(-handshape, -movement, -orientation, -position, -ends_with("_bin"))

df_dtw_zoom_long = left_join(df_dtw_zoom_long, df_dtw_zoom_long_z) %>% 
  left_join(., df_dtw_zoom_long_bin) %>%
  select(-starts_with("total_similarity"))

## for partial residual plot
df_dtw_zoom$handshape_res = residuals(lm(handshape ~ movement + orientation + position, data = df_dtw_zoom))
df_dtw_zoom$movement_res = residuals(lm(movement ~ handshape + orientation + position, data = df_dtw_zoom))
df_dtw_zoom$orientation_res = residuals(lm(orientation ~ handshape + movement + position, data = df_dtw_zoom))
df_dtw_zoom$position_res = residuals(lm(position ~ handshape + movement + orientation, data = df_dtw_zoom))

df_dtw_zoom_long_res = df_dtw_zoom %>%
  pivot_longer(cols = c("handshape_res", "movement_res", "orientation_res", "position_res"), 
               names_to = "feature", 
               values_to = "similarity_res") %>% 
  mutate(feature = factor(sub("_res", "", feature),
                          levels = c("handshape", "movement", "orientation", "position"))) %>% 
  select(!dataset:total_similarity_z)

1.2 Data visualization

1.2.1 correlation for each feature

scp_feature = df_dtw_zoom_long %>%
  ggplot(aes(x = similarity, y = average_distance_xyz)) +
  geom_point(size = 0.8, alpha = 0.7) +
  geom_smooth(method = "lm") +
  labs(x="Similarity score", 
       y="Normalized DTW distance") +
  scale_x_continuous(limits = c(0, 5),
                     breaks = seq(0, 5, 1)) +
  scale_y_continuous(limits = c(0, 1),
                     breaks = seq(0, 1, 0.2)) +
  theme_classic(base_size = 14) +
  theme(axis.text.x = element_text(colour = "black", size = 13),
        axis.text.y = element_text(colour = "black", size = 13),
        axis.title = element_text(face = 'bold'),
        axis.title.x = element_text(vjust = -2),
        axis.title.y = element_text(vjust = 2),
        strip.text = element_text(face = 'bold'),
        legend.position = "none",
        plot.margin = unit(c(1.1,1.1,1.1,1.1), "lines")) +
  facet_wrap(~feature, nrow = 2)

ggplotly(scp_feature)

1.2.2 partial residual plot

scp_feature_res = df_dtw_zoom_long_res %>%
  ggplot(aes(x = similarity_res, y = average_distance_xyz)) +
  geom_point(size = 0.8, alpha = 0.7) +
  geom_smooth(method = "lm") +
  labs(x="Similarity score (residuals)", 
       y="Normalized DTW distance") +
  scale_y_continuous(limits = c(0, 1),
                     breaks = seq(0, 1, 0.2)) +
  theme_classic(base_size = 14) +
  theme(axis.text.x = element_text(colour = "black", size = 13),
        axis.text.y = element_text(colour = "black", size = 13),
        axis.title = element_text(face = 'bold'),
        axis.title.x = element_text(vjust = -2),
        axis.title.y = element_text(vjust = 2),
        strip.text = element_text(face = 'bold'),
        legend.position = "none",
        plot.margin = unit(c(1.1,1.1,1.1,1.1), "lines")) +
  facet_wrap(~feature, nrow = 2)

scp_feature_res

1.2.3 correlation for total similarity score

scp_total = df_dtw_zoom %>%
  ggplot(aes(x = total_similarity, y = average_distance_xyz)) +
  geom_point(size = 1, alpha = 0.7) +
  geom_smooth(method = "lm") +
  labs(x="Total similarity score", 
       y="Normalized DTW distance") +
  scale_x_continuous(limits = c(0, 20),
                     breaks = seq(0, 20, 5)) +
  scale_y_continuous(limits = c(0, 1),
                     breaks = seq(0, 1, 0.2)) +
  theme_classic(base_size = 14) +
  theme(axis.text.x = element_text(colour = "black", size = 13),
        axis.text.y = element_text(colour = "black", size = 13),
        axis.title = element_text(face = 'bold'),
        axis.title.x = element_text(vjust = -2),
        axis.title.y = element_text(vjust = 2),
        strip.text = element_text(face = 'bold'),
        legend.position = "none",
        plot.margin = unit(c(1.1,1.1,1.1,1.1), "lines"))

ggExtra::ggMarginal(scp_total, fill = "lightblue")

# ggplotly(scp_total)

1.2.4 [z] correlation for each feature

scp_feature_z = df_dtw_zoom_long %>%
  ggplot(aes(x = similarity_z, y = average_distance_xyz)) +
  geom_point(size = 0.8, alpha = 0.7) +
  geom_smooth(method = "lm") +
  labs(x="Standardized similarity score", 
       y="Normalized DTW distance") +
  scale_x_continuous(limits = c(-2.5, 2),
                     breaks = seq(-2.5, 2, 1)) +
  scale_y_continuous(limits = c(0, 1),
                     breaks = seq(0, 1, 0.2)) +
  theme_classic(base_size = 14) +
  theme(axis.text.x = element_text(colour = "black", size = 13),
        axis.text.y = element_text(colour = "black", size = 13),
        axis.title = element_text(face = 'bold'),
        axis.title.x = element_text(vjust = -2),
        axis.title.y = element_text(vjust = 2),
        strip.text = element_text(face = 'bold'),
        legend.position = "none",
        plot.margin = unit(c(1.1,1.1,1.1,1.1), "lines")) +
  facet_wrap(~feature, nrow = 2)

scp_feature_z

1.2.5 [z] correlation for total similarity score

scp_total_z = df_dtw_zoom %>%
  ggplot(aes(x = total_similarity_z, y = average_distance_xyz)) +
  geom_point(size = 1, alpha = 0.7) +
  geom_smooth(method = "lm") +
  labs(x="Standardized total similarity score (z)", 
       y="Normalized DTW distance") +
  scale_x_continuous(limits = c(-2.2, 1.5),
                     breaks = seq(-2, 1.5, 0.5)) +
  scale_y_continuous(limits = c(0, 1),
                     breaks = seq(0, 1, 0.2)) +
  theme_classic(base_size = 14) +
  theme(axis.text.x = element_text(colour = "black", size = 13),
        axis.text.y = element_text(colour = "black", size = 13),
        axis.title = element_text(face = 'bold'),
        axis.title.x = element_text(vjust = -2),
        axis.title.y = element_text(vjust = 2),
        strip.text = element_text(face = 'bold'),
        legend.position = "none",
        plot.margin = unit(c(1.1,1.1,1.1,1.1), "lines"))

ggExtra::ggMarginal(scp_total_z, fill = "lightblue")

1.3 Statistical analysis

1.3.1 Check the normality assumption of errors

Linear mixed-effects models assume that the residuals are normally distributed. Let’s check this assumption by plotting the residuals of the models. For this, we will use the check_model() function from the performance package.

model = lmer(average_distance_xyz ~ 1 + (1|pair) + (1|target), 
             data = subset(df_dtw_zoom))
check_model(model)

model_log = lmer(log(average_distance_xyz) ~ 1 + (1|pair) + (1|target), 
                 data = subset(df_dtw_zoom))
check_model(model_log)

Log-normal model returned a sigular fit (i.e., the model is not able to estimate the variance of the random effects for pair). Therefore, we will use linear regression for the analysis.

1.3.2 Correlation for total similarity

x = df_dtw_zoom$total_similarity
y = df_dtw_zoom$average_distance_xyz

cor.test(x, y, method="pearson")

## 
##  Pearson's product-moment correlation
## 
## data:  x and y
## t = -3, df = 98, p-value = 0.003
## alternative hypothesis: true correlation is not equal to 0
## 95 percent confidence interval:
##  -0.466 -0.106
## sample estimates:
##    cor 
## -0.296

cor.test(x, y, method="spearman")

## 
##  Spearman's rank correlation rho
## 
## data:  x and y
## S = 226981, p-value = 0.0002
## alternative hypothesis: true rho is not equal to 0
## sample estimates:
##    rho 
## -0.362

The correlation between the total similarity score and the normalized DTW distance is significant (r = -0.296, p = 0.004), suggesting that the more similar features two gestures have, the smaller the distance is.

1.3.3 LMM for total similarity score

lmer_zoom_total = lmer(average_distance_xyz ~ 
                           total_similarity_z +
                           (1|pair) + (1|target),
                         data = df_dtw_zoom)

summ(lmer_zoom_total, digits = 3)

Observations	100
Dependent variable	average_distance_xyz
Type	Mixed effects linear regression

AIC	-65.149
BIC	-52.123
Pseudo-R² (fixed effects)	0.093
Pseudo-R² (total)	0.162

Fixed Effects
	Est.	S.E.	t val.	d.f.	p
(Intercept)	0.349	0.019	18.201	9.257	0.000
total_similarity_z	-0.051	0.016	-3.225	93.777	0.002
p values calculated using Satterthwaite d.f.

Random Effects
Group	Parameter	Std. Dev.
pair	(Intercept)	0.028
target	(Intercept)	0.034
Residual		0.152

Grouping Variables
Group	# groups	ICC
pair	22	0.031
target	16	0.045

check_model(lmer_zoom_total)

1.3.4 Correlation for each feature

print("Handshape")

## [1] "Handshape"

cor.test(df_dtw_zoom$handshape, df_dtw_zoom$average_distance_xyz, method="spearman")

## 
##  Spearman's rank correlation rho
## 
## data:  df_dtw_zoom$handshape and df_dtw_zoom$average_distance_xyz
## S = 198505, p-value = 0.06
## alternative hypothesis: true rho is not equal to 0
## sample estimates:
##    rho 
## -0.191

print("Orientation")

## [1] "Orientation"

cor.test(df_dtw_zoom$orientation, df_dtw_zoom$average_distance_xyz, method="spearman")

## 
##  Spearman's rank correlation rho
## 
## data:  df_dtw_zoom$orientation and df_dtw_zoom$average_distance_xyz
## S = 206904, p-value = 0.02
## alternative hypothesis: true rho is not equal to 0
## sample estimates:
##    rho 
## -0.242

print("Movement")

## [1] "Movement"

cor.test(df_dtw_zoom$movement, df_dtw_zoom$average_distance_xyz, method="spearman")

## 
##  Spearman's rank correlation rho
## 
## data:  df_dtw_zoom$movement and df_dtw_zoom$average_distance_xyz
## S = 213797, p-value = 0.004
## alternative hypothesis: true rho is not equal to 0
## sample estimates:
##    rho 
## -0.283

print("Position")

## [1] "Position"

cor.test(df_dtw_zoom$position, df_dtw_zoom$average_distance_xyz, method="spearman")

## 
##  Spearman's rank correlation rho
## 
## data:  df_dtw_zoom$position and df_dtw_zoom$average_distance_xyz
## S = 250467, p-value = 0.0000001
## alternative hypothesis: true rho is not equal to 0
## sample estimates:
##    rho 
## -0.503

1.3.5 LMM for each feature

### each feature
lmer_zoom_shape = lmer(average_distance_xyz ~ 
                 handshape_z + (1|pair) + (1|target),
               data = df_dtw_zoom)
summ(lmer_zoom_shape, digits = 3)

Observations	100
Dependent variable	average_distance_xyz
Type	Mixed effects linear regression

AIC	-56.998
BIC	-43.972
Pseudo-R² (fixed effects)	0.015
Pseudo-R² (total)	0.060

Fixed Effects
	Est.	S.E.	t val.	d.f.	p
(Intercept)	0.351	0.019	18.821	7.653	0.000
handshape_z	-0.020	0.016	-1.238	96.778	0.219
p values calculated using Satterthwaite d.f.

Random Effects
Group	Parameter	Std. Dev.
pair	(Intercept)	0.020
target	(Intercept)	0.029
Residual		0.161

Grouping Variables
Group	# groups	ICC
pair	22	0.015
target	16	0.031

lmer_zoom_movement = lmer(average_distance_xyz ~ 
                             movement_z + (1|pair) + (1|target),
                           data = df_dtw_zoom)
summ(lmer_zoom_movement, digits = 3)

Observations	100
Dependent variable	average_distance_xyz
Type	Mixed effects linear regression

AIC	-61.372
BIC	-48.346
Pseudo-R² (fixed effects)	0.058
Pseudo-R² (total)	0.113

Fixed Effects
	Est.	S.E.	t val.	d.f.	p
(Intercept)	0.351	0.019	18.637	8.177	0.000
movement_z	-0.040	0.016	-2.481	95.894	0.015
p values calculated using Satterthwaite d.f.

Random Effects
Group	Parameter	Std. Dev.
pair	(Intercept)	0.022
target	(Intercept)	0.032
Residual		0.156

Grouping Variables
Group	# groups	ICC
pair	22	0.018
target	16	0.041

lmer_zoom_orientation = lmer(average_distance_xyz ~ 
                                orientation_z + (1|pair) + (1|target),
                              data = df_dtw_zoom)
summ(lmer_zoom_orientation, digits = 3)

Observations	100
Dependent variable	average_distance_xyz
Type	Mixed effects linear regression

AIC	-59.376
BIC	-46.350
Pseudo-R² (fixed effects)	0.039
Pseudo-R² (total)	0.101

Fixed Effects
	Est.	S.E.	t val.	d.f.	p
(Intercept)	0.350	0.019	18.150	8.251	0.000
orientation_z	-0.033	0.016	-2.030	94.347	0.045
p values calculated using Satterthwaite d.f.

Random Effects
Group	Parameter	Std. Dev.
pair	(Intercept)	0.029
target	(Intercept)	0.029
Residual		0.158

Grouping Variables
Group	# groups	ICC
pair	22	0.032
target	16	0.033

lmer_zoom_position = lmer(average_distance_xyz ~ 
                              position_z + (1|pair) + (1|target),
                            data = df_dtw_zoom)
summ(lmer_zoom_position, digits = 3)

Observations	100
Dependent variable	average_distance_xyz
Type	Mixed effects linear regression

AIC	-77.154
BIC	-64.129
Pseudo-R² (fixed effects)	0.195
Pseudo-R² (total)	0.248

Fixed Effects
	Est.	S.E.	t val.	d.f.	p
(Intercept)	0.349	0.018	19.804	9.665	0.000
position_z	-0.073	0.015	-4.944	96.571	0.000
p values calculated using Satterthwaite d.f.

Random Effects
Group	Parameter	Std. Dev.
pair	(Intercept)	0.018
target	(Intercept)	0.033
Residual		0.144

Grouping Variables
Group	# groups	ICC
pair	22	0.016
target	16	0.050

### all features
lmer_zoom_feature = lmer(average_distance_xyz ~ 
                            handshape_z + movement_z + orientation_z + position_z +
                            (1|pair) + (1|target),
                          data = df_dtw_zoom)

summ(lmer_zoom_feature, digits = 3)

Observations	100
Dependent variable	average_distance_xyz
Type	Mixed effects linear regression

AIC	-53.097
BIC	-32.255
Pseudo-R² (fixed effects)	0.192
Pseudo-R² (total)	0.244

Fixed Effects
	Est.	S.E.	t val.	d.f.	p
(Intercept)	0.349	0.018	19.525	9.385	0.000
handshape_z	-0.001	0.021	-0.025	94.887	0.980
movement_z	0.007	0.021	0.335	94.358	0.738
orientation_z	-0.003	0.024	-0.143	92.887	0.887
position_z	-0.076	0.019	-4.013	94.250	0.000
p values calculated using Satterthwaite d.f.

Random Effects
Group	Parameter	Std. Dev.
pair	(Intercept)	0.019
target	(Intercept)	0.033
Residual		0.146

Grouping Variables
Group	# groups	ICC
pair	22	0.016
target	16	0.049

cov2cor(vcov(lmer_zoom_feature))

## 5 x 5 Matrix of class "corMatrix"
##               (Intercept) handshape_z movement_z orientation_z position_z
## (Intercept)       1.00000    -0.00179    -0.0126        0.0102     0.0171
## handshape_z      -0.00179     1.00000    -0.0582       -0.6197     0.0693
## movement_z       -0.01261    -0.05821     1.0000       -0.3121    -0.4659
## orientation_z     0.01016    -0.61973    -0.3121        1.0000    -0.1506
## position_z        0.01705     0.06926    -0.4659       -0.1506     1.0000

Regressions on each feature show a significant negative association between DTW distance and movement, orientation, and position (handshape didn’t reach significance). However, the model with all features as fixed effects shows such significant negative correlation only for position.

To check why the effects “disappeared”, we examined the correlation of fixed effects and found a strong negative correlation between handshape and orientation (-0.6) and between movement and position (-0.47). This suggests that when the slope estimate for orientation is more extremely negative, the slope estimate for handshape becomes flatter (or more positive). This is likely due to the fact that handshape and orientation are both reflected in relative finger tip positions. Similar relationship was found for movement and position, which is likely due to the fact that they are both reflected in the wrist positions.

Again, this suggests a need for developing a different approach to capturing handshape similarity independently of hand orientation.

2 Revised DTW pipeline for Zoom dataset

A qualitative check on incongruent cases where a pair of gestures was annotated as similar but it’s DTW distance was large revealed that the majority of such cases is because of mirrored movements. To tackle this issue, for gestures where speaker A and B used the opposing hands (e.g., speaker used left-hand and speaker B right-hand), we will calculate DTW distance for both original and flipped videos and take the minimum of the two.

2.1 Data wrangling

2.1.1 Load data

### Load data
df_dtw_zoom_v2 = read_csv("data/dtw_distance_zoom_mirrored.csv") %>% 
  rename(target = target_2) %>%
  select(comparison_id, pair, target, average_distance, average_distance_xyz, hands_dtw) %>%
  mutate(dataset = "zoom",
         pair = as.factor(pair),
         target = as.factor(target))

df_sim_coding_zoom = read_csv("data/similarity_coding_zoom.csv") %>% 
  rename(position = location) %>% 
  mutate(total_similarity = handshape + movement + orientation + position,
         ### binary coding
         handshape_bin = ifelse(handshape >= 4, 1, 0),
         movement_bin = ifelse(movement >= 4, 1, 0),
         orientation_bin = ifelse(orientation >= 4, 1, 0),
         position_bin = ifelse(position >= 4, 1, 0),
         n_features = handshape_bin + movement_bin + orientation_bin + position_bin,
         handshape_bin = factor(handshape_bin, 
                                levels = c(0, 1), 
                                labels = c("not similar", "similar")),
         movement_bin = factor(movement_bin,
                                levels = c(0, 1), 
                                labels = c("not similar", "similar")),
         orientation_bin = factor(orientation_bin,
                                   levels = c(0, 1), 
                                   labels = c("not similar", "similar")),
         position_bin = factor(position_bin,
                                levels = c(0, 1), 
                                labels = c("not similar", "similar")),
         ### z-score
         handshape_z = scale(handshape)[,1],
         movement_z = scale(movement)[,1],
         orientation_z = scale(orientation)[,1],
         position_z = scale(position)[,1],
         total_similarity_z = scale(total_similarity)[,1]) %>% 
  select(-notes)

### combine dtw and coding data for zoom dataset
df_dtw_zoom_v2 = right_join(df_dtw_zoom_v2, df_sim_coding_zoom)

### convert to long format for data visualization
df_dtw_zoom_v2_long = df_dtw_zoom_v2 %>%
  pivot_longer(cols = c("handshape", "movement", "orientation", "position"), 
               names_to = "feature", 
               values_to = "similarity") %>% 
  select(-ends_with("_z"), -ends_with("_bin"))

df_dtw_zoom_v2_long_bin = df_dtw_zoom_v2 %>%
  pivot_longer(cols = c("handshape_bin", "movement_bin", "orientation_bin", "position_bin"), 
               names_to = "feature", 
               values_to = "similarity_bin") %>% 
  mutate(feature = factor(sub("_bin", "", feature),
                          levels = c("handshape", "movement", "orientation", "position"))) %>% 
  select(-handshape, -movement, -orientation, -position, -ends_with("_z"))

df_dtw_zoom_v2_long_z = df_dtw_zoom_v2 %>%
  pivot_longer(cols = c("handshape_z", "movement_z", "orientation_z", "position_z"), 
               names_to = "feature", 
               values_to = "similarity_z") %>% 
  mutate(feature = factor(sub("_z", "", feature),
                          levels = c("handshape", "movement", "orientation", "position"))) %>% 
  select(-handshape, -movement, -orientation, -position, -ends_with("_bin"))

df_dtw_zoom_v2_long = left_join(df_dtw_zoom_v2_long, df_dtw_zoom_v2_long_z) %>% 
  left_join(., df_dtw_zoom_v2_long_bin) %>%
  select(-starts_with("total_similarity"))

## for partial residual plot
df_dtw_zoom_v2$handshape_res = residuals(lm(handshape ~ movement + orientation + position, data = df_dtw_zoom_v2))
df_dtw_zoom_v2$movement_res = residuals(lm(movement ~ handshape + orientation + position, data = df_dtw_zoom_v2))
df_dtw_zoom_v2$orientation_res = residuals(lm(orientation ~ handshape + movement + position, data = df_dtw_zoom_v2))
df_dtw_zoom_v2$position_res = residuals(lm(position ~ handshape + movement + orientation, data = df_dtw_zoom_v2))

df_dtw_zoom_v2_long_res = df_dtw_zoom_v2 %>%
  pivot_longer(cols = c("handshape_res", "movement_res", "orientation_res", "position_res"), 
               names_to = "feature", 
               values_to = "similarity_res") %>% 
  mutate(feature = factor(sub("_res", "", feature),
                          levels = c("handshape", "movement", "orientation", "position"))) %>% 
  select(!dataset:total_similarity_z)

2.2 Data visualization

2.2.1 correlation for each feature

scp_feature = df_dtw_zoom_v2_long %>%
  ggplot(aes(x = similarity, y = average_distance_xyz)) +
  geom_point(size = 0.8, alpha = 0.7) +
  geom_smooth(method = "lm") +
  labs(x="Similarity score", 
       y="Normalized DTW distance") +
  scale_x_continuous(limits = c(0, 5),
                     breaks = seq(0, 5, 1)) +
  scale_y_continuous(limits = c(0, 0.8),
                     breaks = seq(0, 0.8, 0.2)) +
  theme_classic(base_size = 14) +
  theme(axis.text.x = element_text(colour = "black", size = 13),
        axis.text.y = element_text(colour = "black", size = 13),
        axis.title = element_text(face = 'bold'),
        axis.title.x = element_text(vjust = -2),
        axis.title.y = element_text(vjust = 2),
        strip.text = element_text(face = 'bold'),
        legend.position = "none",
        plot.margin = unit(c(1.1,1.1,1.1,1.1), "lines")) +
  facet_wrap(~feature, nrow = 2)

ggplotly(scp_feature)

2.2.2 partial residual plot

scp_feature_res = df_dtw_zoom_v2_long_res %>%
  ggplot(aes(x = similarity_res, y = average_distance_xyz)) +
  geom_point(size = 0.8, alpha = 0.7) +
  geom_smooth(method = "lm") +
  labs(x="Similarity score (residuals)", 
       y="Normalized DTW distance") +
  scale_y_continuous(limits = c(0, 0.8),
                     breaks = seq(0, 0.8, 0.2)) +
  theme_classic(base_size = 14) +
  theme(axis.text.x = element_text(colour = "black", size = 13),
        axis.text.y = element_text(colour = "black", size = 13),
        axis.title = element_text(face = 'bold'),
        axis.title.x = element_text(vjust = -2),
        axis.title.y = element_text(vjust = 2),
        strip.text = element_text(face = 'bold'),
        legend.position = "none",
        plot.margin = unit(c(1.1,1.1,1.1,1.1), "lines")) +
  facet_wrap(~feature, nrow = 2)

scp_feature_res

2.2.3 correlation for total similarity score

scp_total = df_dtw_zoom_v2 %>%
  ggplot(aes(x = total_similarity, y = average_distance_xyz)) +
  geom_point(size = 1, alpha = 0.7) +
  geom_smooth(method = "lm") +
  labs(x="Total similarity score", 
       y="Normalized DTW distance") +
  scale_x_continuous(limits = c(0, 20),
                     breaks = seq(0, 20, 5)) +
  scale_y_continuous(limits = c(0, 0.8),
                     breaks = seq(0, 0.8, 0.2)) +
  theme_classic(base_size = 14) +
  theme(axis.text.x = element_text(colour = "black", size = 13),
        axis.text.y = element_text(colour = "black", size = 13),
        axis.title = element_text(face = 'bold'),
        axis.title.x = element_text(vjust = -2),
        axis.title.y = element_text(vjust = 2),
        strip.text = element_text(face = 'bold'),
        legend.position = "none",
        plot.margin = unit(c(1.1,1.1,1.1,1.1), "lines"))

ggExtra::ggMarginal(scp_total, fill = "lightblue")

# ggplotly(scp_total)

2.2.4 [z] correlation for each feature

scp_feature_z = df_dtw_zoom_v2_long %>%
  ggplot(aes(x = similarity_z, y = average_distance_xyz)) +
  geom_point(size = 0.8, alpha = 0.7) +
  geom_smooth(method = "lm") +
  labs(x="Standardized similarity score", 
       y="Normalized DTW distance") +
  scale_x_continuous(limits = c(-2.5, 2),
                     breaks = seq(-2.5, 2, 1)) +
  scale_y_continuous(limits = c(0, 0.8),
                     breaks = seq(0, 0.8, 0.2)) +
  theme_classic(base_size = 14) +
  theme(axis.text.x = element_text(colour = "black", size = 13),
        axis.text.y = element_text(colour = "black", size = 13),
        axis.title = element_text(face = 'bold'),
        axis.title.x = element_text(vjust = -2),
        axis.title.y = element_text(vjust = 2),
        strip.text = element_text(face = 'bold'),
        legend.position = "none",
        plot.margin = unit(c(1.1,1.1,1.1,1.1), "lines")) +
  facet_wrap(~feature, nrow = 2)

scp_feature_z

2.2.5 [z] correlation for total similarity score

scp_total_z = df_dtw_zoom_v2 %>%
  ggplot(aes(x = total_similarity_z, y = average_distance_xyz)) +
  geom_point(size = 1, alpha = 0.7) +
  geom_smooth(method = "lm") +
  labs(x="Standardized total similarity score (z)", 
       y="Normalized DTW distance") +
  scale_x_continuous(limits = c(-2.2, 1.5),
                     breaks = seq(-2, 1.5, 0.5)) +
  scale_y_continuous(limits = c(0, 0.8),
                     breaks = seq(0, 0.8, 0.2)) +
  theme_classic(base_size = 14) +
  theme(axis.text.x = element_text(colour = "black", size = 13),
        axis.text.y = element_text(colour = "black", size = 13),
        axis.title = element_text(face = 'bold'),
        axis.title.x = element_text(vjust = -2),
        axis.title.y = element_text(vjust = 2),
        strip.text = element_text(face = 'bold'),
        legend.position = "none",
        plot.margin = unit(c(1.1,1.1,1.1,1.1), "lines"))

ggExtra::ggMarginal(scp_total_z, fill = "lightblue")

2.3 Statistical analysis

2.3.1 Check the normality assumption of errors

model = lmer(average_distance_xyz ~ 1 + (1|pair), 
             data = subset(df_dtw_zoom_v2))
check_model(model)

model_log = lmer(log(average_distance_xyz) ~ 1 + (1|pair), 
                 data = subset(df_dtw_zoom_v2))
check_model(model_log)

Models including random intercepts for dyads and items returned a singular fit, most likely because of the small sample size. After removing the by-item random intercept, the model converged only for the linear model. Therefore, we will use linear regression for the analysis.

2.3.2 Correlation for total similarity

x = df_dtw_zoom_v2$total_similarity
y = df_dtw_zoom_v2$average_distance_xyz

cor.test(x, y, method="pearson")

## 
##  Pearson's product-moment correlation
## 
## data:  x and y
## t = -6, df = 98, p-value = 0.00000002
## alternative hypothesis: true correlation is not equal to 0
## 95 percent confidence interval:
##  -0.654 -0.366
## sample estimates:
##    cor 
## -0.525

cor.test(x, y, method="spearman")

## 
##  Spearman's rank correlation rho
## 
## data:  x and y
## S = 258048, p-value = 0.000000003
## alternative hypothesis: true rho is not equal to 0
## sample estimates:
##    rho 
## -0.548

2.3.3 LMM for total similarity score

lmer_zoom_total = lmer(average_distance_xyz ~ 
                           total_similarity_z +
                           (1|pair),
                         data = df_dtw_zoom_v2)

summ(lmer_zoom_total, digits = 3)

Observations	100
Dependent variable	average_distance_xyz
Type	Mixed effects linear regression

AIC	-133.125
BIC	-122.704
Pseudo-R² (fixed effects)	0.278
Pseudo-R² (total)	0.328

Fixed Effects
	Est.	S.E.	t val.	d.f.	p
(Intercept)	0.309	0.013	23.038	14.433	0.000
total_similarity_z	-0.070	0.011	-6.291	93.911	0.000
p values calculated using Satterthwaite d.f.

Random Effects
Group	Parameter	Std. Dev.
pair	(Intercept)	0.030
Residual		0.109

Grouping Variables
Group	# groups	ICC
pair	22	0.068

check_model(lmer_zoom_total)

2.3.4 Correlation for each feature

print("Handshape")

## [1] "Handshape"

cor.test(df_dtw_zoom_v2$handshape, df_dtw_zoom_v2$average_distance_xyz, method="spearman")

## 
##  Spearman's rank correlation rho
## 
## data:  df_dtw_zoom_v2$handshape and df_dtw_zoom_v2$average_distance_xyz
## S = 227495, p-value = 0.0002
## alternative hypothesis: true rho is not equal to 0
## sample estimates:
##    rho 
## -0.365

print("Orientation")

## [1] "Orientation"

cor.test(df_dtw_zoom_v2$orientation, df_dtw_zoom_v2$average_distance_xyz, method="spearman")

## 
##  Spearman's rank correlation rho
## 
## data:  df_dtw_zoom_v2$orientation and df_dtw_zoom_v2$average_distance_xyz
## S = 223012, p-value = 0.0006
## alternative hypothesis: true rho is not equal to 0
## sample estimates:
##    rho 
## -0.338

print("Movement")

## [1] "Movement"

cor.test(df_dtw_zoom_v2$movement, df_dtw_zoom_v2$average_distance_xyz, method="spearman")

## 
##  Spearman's rank correlation rho
## 
## data:  df_dtw_zoom_v2$movement and df_dtw_zoom_v2$average_distance_xyz
## S = 233120, p-value = 0.00004
## alternative hypothesis: true rho is not equal to 0
## sample estimates:
##    rho 
## -0.399

print("Position")

## [1] "Position"

cor.test(df_dtw_zoom_v2$position, df_dtw_zoom_v2$average_distance_xyz, method="spearman")

## 
##  Spearman's rank correlation rho
## 
## data:  df_dtw_zoom_v2$position and df_dtw_zoom_v2$average_distance_xyz
## S = 281737, p-value = 0.000000000000002
## alternative hypothesis: true rho is not equal to 0
## sample estimates:
##    rho 
## -0.691

2.3.5 LMM for each feature

### each feature
lmer_zoom_shape = lmer(average_distance_xyz ~ 
                 handshape_z + (1|pair),
               data = df_dtw_zoom_v2)
summ(lmer_zoom_shape, digits = 3)

Observations	100
Dependent variable	average_distance_xyz
Type	Mixed effects linear regression

AIC	-111.445
BIC	-101.025
Pseudo-R² (fixed effects)	0.105
Pseudo-R² (total)	0.139

Fixed Effects
	Est.	S.E.	t val.	d.f.	p
(Intercept)	0.309	0.014	22.090	12.438	0.000
handshape_z	-0.043	0.013	-3.429	96.599	0.001
p values calculated using Satterthwaite d.f.

Random Effects
Group	Parameter	Std. Dev.
pair	(Intercept)	0.025
Residual		0.123

Grouping Variables
Group	# groups	ICC
pair	22	0.038

lmer_zoom_movement = lmer(average_distance_xyz ~ 
                             movement_z + (1|pair),
                           data = df_dtw_zoom_v2)
summ(lmer_zoom_movement, digits = 3)

Observations	100
Dependent variable	average_distance_xyz
Type	Mixed effects linear regression

AIC	-118.297
BIC	-107.876
Pseudo-R² (fixed effects)	0.164
Pseudo-R² (total)	0.196

Fixed Effects
	Est.	S.E.	t val.	d.f.	p
(Intercept)	0.310	0.014	22.974	13.656	0.000
movement_z	-0.054	0.012	-4.438	96.736	0.000
p values calculated using Satterthwaite d.f.

Random Effects
Group	Parameter	Std. Dev.
pair	(Intercept)	0.024
Residual		0.119

Grouping Variables
Group	# groups	ICC
pair	22	0.038

lmer_zoom_orientation = lmer(average_distance_xyz ~ 
                                orientation_z + (1|pair),
                              data = df_dtw_zoom_v2)
summ(lmer_zoom_orientation, digits = 3)

Observations	100
Dependent variable	average_distance_xyz
Type	Mixed effects linear regression

AIC	-112.806
BIC	-102.385
Pseudo-R² (fixed effects)	0.120
Pseudo-R² (total)	0.182

Fixed Effects
	Est.	S.E.	t val.	d.f.	p
(Intercept)	0.309	0.015	20.676	13.725	0.000
orientation_z	-0.046	0.012	-3.723	95.334	0.000
p values calculated using Satterthwaite d.f.

Random Effects
Group	Parameter	Std. Dev.
pair	(Intercept)	0.033
Residual		0.121

Grouping Variables
Group	# groups	ICC
pair	22	0.071

lmer_zoom_position = lmer(average_distance_xyz ~ 
                              position_z + (1|pair),
                            data = df_dtw_zoom_v2)
summ(lmer_zoom_position, digits = 3)

Observations	100
Dependent variable	average_distance_xyz
Type	Mixed effects linear regression

AIC	-157.211
BIC	-146.790
Pseudo-R² (fixed effects)	0.438
Pseudo-R² (total)	0.462

Fixed Effects
	Est.	S.E.	t val.	d.f.	p
(Intercept)	0.309	0.011	27.471	15.242	0.000
position_z	-0.088	0.010	-8.785	97.982	0.000
p values calculated using Satterthwaite d.f.

Random Effects
Group	Parameter	Std. Dev.
pair	(Intercept)	0.021
Residual		0.098

Grouping Variables
Group	# groups	ICC
pair	22	0.044

### all features
lmer_zoom_feature = lmer(average_distance_xyz ~ 
                            handshape_z + movement_z + orientation_z + position_z +
                            (1|pair),
                          data = df_dtw_zoom_v2)

summ(lmer_zoom_feature, digits = 3)

Observations	100
Dependent variable	average_distance_xyz
Type	Mixed effects linear regression

AIC	-134.496
BIC	-116.260
Pseudo-R² (fixed effects)	0.451
Pseudo-R² (total)	0.475

Fixed Effects
	Est.	S.E.	t val.	d.f.	p
(Intercept)	0.308	0.011	27.664	15.178	0.000
handshape_z	-0.025	0.014	-1.802	94.699	0.075
movement_z	0.002	0.014	0.128	94.975	0.899
orientation_z	0.008	0.016	0.519	94.910	0.605
position_z	-0.085	0.012	-6.859	94.981	0.000
p values calculated using Satterthwaite d.f.

Random Effects
Group	Parameter	Std. Dev.
pair	(Intercept)	0.020
Residual		0.097

Grouping Variables
Group	# groups	ICC
pair	22	0.042

cov2cor(vcov(lmer_zoom_feature))

## 5 x 5 Matrix of class "corMatrix"
##               (Intercept) handshape_z movement_z orientation_z position_z
## (Intercept)       1.00000     0.00177    -0.0252        0.0265     0.0148
## handshape_z       0.00177     1.00000    -0.0559       -0.6135     0.0481
## movement_z       -0.02519    -0.05590     1.0000       -0.3260    -0.4531
## orientation_z     0.02645    -0.61347    -0.3260        1.0000    -0.1393
## position_z        0.01483     0.04805    -0.4531       -0.1393     1.0000

Regressions on each feature show a significant negative association between DTW distance for all features. However, the model with all features as fixed effects shows such significant negative correlation only for position.

To check why the effects “disappeared”, we examined the correlation of fixed effects and found a strong negative correlation between handshape and orientation (-0.6) and between movement and position (-0.45). This suggests that when the slope estimate for orientation is more extremely negative, the slope estimate for handshape becomes flatter (or more positive). This is likely due to the fact that handshape and orientation are both reflected in relative finger tip positions. Similar relationship was found for movement and position, which is likely due to the fact that they are both reflected in the wrist positions.

3 Session Info

sessionInfo()

## R version 4.4.0 (2024-04-24 ucrt)
## Platform: x86_64-w64-mingw32/x64
## Running under: Windows 11 x64 (build 22631)
## 
## Matrix products: default
## 
## 
## locale:
## [1] LC_COLLATE=English_United States.utf8 
## [2] LC_CTYPE=English_United States.utf8   
## [3] LC_MONETARY=English_United States.utf8
## [4] LC_NUMERIC=C                          
## [5] LC_TIME=English_United States.utf8    
## 
## time zone: Europe/Amsterdam
## tzcode source: internal
## 
## attached base packages:
## [1] stats     graphics  grDevices utils     datasets  methods   base     
## 
## other attached packages:
##  [1] kableExtra_1.4.0   sjPlot_2.8.16      performance_0.11.0 glmmTMB_1.1.9     
##  [5] jtools_2.2.2       lmerTest_3.1-3     lme4_1.1-35.3      Matrix_1.7-0      
##  [9] plotrix_3.8-4      patchwork_1.2.0    plotly_4.10.4      ggtext_0.1.2      
## [13] lubridate_1.9.3    forcats_1.0.0      stringr_1.5.1      dplyr_1.1.4       
## [17] purrr_1.0.2        readr_2.1.5        tidyr_1.3.1        tibble_3.2.1      
## [21] ggplot2_3.5.1      tidyverse_2.0.0   
## 
## loaded via a namespace (and not attached):
##  [1] rlang_1.1.4         magrittr_2.0.3      compiler_4.4.0     
##  [4] mgcv_1.9-1          systemfonts_1.1.0   vctrs_0.6.5        
##  [7] pkgconfig_2.0.3     crayon_1.5.2        fastmap_1.2.0      
## [10] labeling_0.4.3      pander_0.6.5        utf8_1.2.4         
## [13] promises_1.3.0      rmarkdown_2.27      tzdb_0.4.0         
## [16] nloptr_2.0.3        bit_4.0.5           xfun_0.44          
## [19] cachem_1.1.0        jsonlite_2.0.0      highr_0.11         
## [22] later_1.3.2         sjmisc_2.8.10       ggeffects_1.6.0    
## [25] parallel_4.4.0      R6_2.5.1            bslib_0.7.0        
## [28] stringi_1.8.4       boot_1.3-30         jquerylib_0.1.4    
## [31] numDeriv_2016.8-1.1 estimability_1.5.1  Rcpp_1.0.13        
## [34] knitr_1.46          httpuv_1.6.15       splines_4.4.0      
## [37] timechange_0.3.0    tidyselect_1.2.1    rstudioapi_0.16.0  
## [40] yaml_2.3.8          TMB_1.9.11          miniUI_0.1.1.1     
## [43] sjlabelled_1.2.0    lattice_0.22-6      bayestestR_0.13.2  
## [46] shiny_1.8.1.1       withr_3.0.1         coda_0.19-4.1      
## [49] evaluate_0.23       xml2_1.3.6          pillar_1.9.0       
## [52] insight_0.19.11     generics_0.1.3      vroom_1.6.5        
## [55] hms_1.1.3           munsell_0.5.1       scales_1.3.0       
## [58] minqa_1.2.7         xtable_1.8-4        glue_1.8.0         
## [61] emmeans_1.10.3      lazyeval_0.2.2      tools_4.4.0        
## [64] see_0.8.4           data.table_1.16.2   mvtnorm_1.3-1      
## [67] grid_4.4.0          crosstalk_1.2.1     datawizard_0.10.0  
## [70] colorspace_2.1-1    nlme_3.1-164        cli_3.6.3          
## [73] fansi_1.0.6         viridisLite_0.4.2   svglite_2.1.3      
## [76] sjstats_0.19.0      gtable_0.3.5        sass_0.4.9         
## [79] digest_0.6.37       ggrepel_0.9.6       htmlwidgets_1.6.4  
## [82] farver_2.1.2        htmltools_0.5.8.1   lifecycle_1.0.4    
## [85] httr_1.4.7          mime_0.12           gridtext_0.1.5     
## [88] ggExtra_0.10.1      bit64_4.0.5         MASS_7.3-60.2

DTW validation analysis

Last updated: 2025-05-16